Vehicle Detection

In [1]:
import numpy as np
from pathlib import Path
import matplotlib.pyplot as plt
import cv2
import time

from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from skimage.feature import hog
from scipy.ndimage.measurements import label

from collections import deque

%matplotlib inline

ROOT_PATH = Path('/home/downloads/carnd-p5-vehicle-detection')

HOG Features

In [225]:
img_car_path = ROOT_PATH/'images/examples/car.png'
img_notcar_path = ROOT_PATH/'images/examples/notcar.png'

img_car = plt.imread(img_car_path.as_posix())
img_notcar = plt.imread(img_notcar_path.as_posix())

print('Image shape', img_car.shape)
plt.imshow(img_car)
plt.show()
plt.imshow(img_notcar)
plt.show()
Image shape (64, 64, 3)
In [3]:
def vis_hog_features(img, orient, pix_per_cell, cell_per_block, vis=True,
                    feature_vec=False):
    
    hog_img = None
    if vis:
        hog_features, hog_img = hog(img, orientations=orient, 
                                pixels_per_cell=(pix_per_cell, pix_per_cell),
                                cells_per_block=(cell_per_block, cell_per_block),
                                visualize=vis, feature_vector=feature_vec, block_norm='L2-Hys')
    else:
        hog_features = hog(img, orientations=orient, 
                        pixels_per_cell=(pix_per_cell, pix_per_cell),
                        cells_per_block=(cell_per_block, cell_per_block),
                        visualize=vis, feature_vector=feature_vec, block_norm='L2-Hys')
    return hog_features, hog_img
In [218]:
gray = cv2.cvtColor(img_car, cv2.COLOR_RGB2GRAY)
features, hog_img = vis_hog_features(gray, orient=9, pix_per_cell=16, cell_per_block=2, vis=True, feature_vec=False)
In [219]:
# Plot the examples
fig = plt.figure(figsize=(12, 9))
plt.subplot(121)
plt.imshow(img_car, cmap='gray')
plt.title('Example Car Image')
plt.subplot(122)
plt.imshow(hog_img, cmap='gray')
plt.title('HOG Visualization')
Out[219]:
Text(0.5,1,'HOG Visualization')

Color space

In [10]:
test_image_path = ROOT_PATH/'images/test_images/test1.jpg'
test_image = read_image(test_image_path.as_posix())
In [11]:
img = convert_color_rgb(test_image, 'RGB')
fig, axes = plt.subplots(1, 3, figsize=(16, 12))
for i, ax in enumerate(axes):
    ax.imshow(img[:, :, i])
In [12]:
img = convert_color_rgb(test_image, 'YUV')
fig, axes = plt.subplots(1, 3, figsize=(16, 12))
for i, ax in enumerate(axes):
    ax.imshow(img[:, :, i])
In [13]:
img = convert_color_rgb(test_image, 'HSV')
fig, axes = plt.subplots(1, 3, figsize=(16, 12))
for i, ax in enumerate(axes):
    ax.imshow(img[:, :, i])
In [14]:
img = convert_color_rgb(test_image, 'HLS')
fig, axes = plt.subplots(1, 3, figsize=(16, 12))
for i, ax in enumerate(axes):
    ax.imshow(img[:, :, i])
In [15]:
img = convert_color_rgb(test_image, 'YCrCb')
fig, axes = plt.subplots(1, 3, figsize=(16, 12))
for i, ax in enumerate(axes):
    ax.imshow(img[:, :, i])
In [16]:
img = convert_color_rgb(test_image, 'LAB')
fig, axes = plt.subplots(1, 3, figsize=(16, 12))
for i, ax in enumerate(axes):
    ax.imshow(img[:, :, i])
In [247]:
img_car_ch = img_car[:, :, 0]
features, hog_img = vis_hog_features(img_car_ch, orient=9, pix_per_cell=16, cell_per_block=2, vis=True, feature_vec=False)

# Plot the examples
fig = plt.figure(figsize=(15, 10))
plt.subplot(231)
plt.imshow(img_car_ch)
plt.title('Channel 1')
plt.subplot(234)
plt.imshow(hog_img, cmap='gray')
plt.title('HOG Visualization')

img_car_ch = img_car[:, :, 1]
features, hog_img = vis_hog_features(img_car_ch, orient=9, pix_per_cell=16, cell_per_block=2, vis=True, feature_vec=False)
plt.subplot(232)
plt.imshow(img_car_ch)
plt.title('Channel 2')
plt.subplot(235)
plt.imshow(hog_img, cmap='gray')
plt.title('HOG Visualization')

img_car_ch = img_car[:, :, 2]
features, hog_img = vis_hog_features(img_car_ch, orient=9, pix_per_cell=16, cell_per_block=2, vis=True, feature_vec=False)
plt.subplot(233)
plt.imshow(img_car_ch)
plt.title('Channel 3')
plt.subplot(236)
plt.imshow(hog_img, cmap='gray')
plt.title('HOG Visualization')
Out[247]:
Text(0.5,1,'HOG Visualization')
In [248]:
img_car_cvt = cv2.cvtColor(img_car, cv2.COLOR_RGB2YUV)
img_car_ch = img_car[:, :, 0]
features, hog_img = vis_hog_features(img_car_ch, orient=9, pix_per_cell=16, cell_per_block=2, vis=True, feature_vec=False)

# Plot the examples
fig = plt.figure(figsize=(15, 10))
plt.subplot(231)
plt.imshow(img_car_ch)
plt.title('Channel 1')
plt.subplot(234)
plt.imshow(hog_img, cmap='gray')
plt.title('HOG Visualization')

img_car_ch = img_car_cvt[:, :, 1]
features, hog_img = vis_hog_features(img_car_ch, orient=9, pix_per_cell=16, cell_per_block=2, vis=True, feature_vec=False)
plt.subplot(232)
plt.imshow(img_car_ch)
plt.title('Channel 2')
plt.subplot(235)
plt.imshow(hog_img, cmap='gray')
plt.title('HOG Visualization')

img_car_ch = img_car_cvt[:, :, 2]
features, hog_img = vis_hog_features(img_car_ch, orient=9, pix_per_cell=16, cell_per_block=2, vis=True, feature_vec=False)
plt.subplot(233)
plt.imshow(img_car_ch)
plt.title('Channel 3')
plt.subplot(236)
plt.imshow(hog_img, cmap='gray')
plt.title('HOG Visualization')
Out[248]:
Text(0.5,1,'HOG Visualization')

Features Classify

Three features are considered:

  1. Spatial bins
  2. histogram
  3. HOG
In [7]:
def read_image(img_path):
    if img_path.endswith('.png'):
        img = plt.imread(img_path)*255
        img = np.clip(img, 0, 255).astype(np.uint8)
    else:
        img = plt.imread(img_path)
    
    return img

def bin_spatial(img, size=(32, 32)):
    features = cv2.resize(img, size).ravel() 
    return features

def color_hist(img, nbins=32):    #bins_range=(0, 256)
    # Compute the histogram of the color channels separately
    channel1_hist = np.histogram(img[:,:,0], bins=nbins)
    channel2_hist = np.histogram(img[:,:,1], bins=nbins)
    channel3_hist = np.histogram(img[:,:,2], bins=nbins)
    # Concatenate the histograms into a single feature vector
    hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
    # Return the individual histograms, bin_centers and feature vector
    return hist_features

def convert_color_rgb(image, cspace='RGB'):
    if cspace != 'RGB':
        if cspace == 'HSV':
            cvt_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
        elif cspace == 'LUV':
            cvt_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
        elif cspace == 'HLS':
            cvt_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
        elif cspace == 'YUV':
            cvt_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
        elif cspace == 'YCrCb':
            cvt_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
        elif cspace == 'LAB':
            cvt_image = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)
    else:
        cvt_image = np.copy(image)
    
    return cvt_image

def extract_hog_features(img, orient, pix_per_cell, cell_per_block, feature_vec=True):
    hog_features = hog(img, orientations=orient, 
                pixels_per_cell=(pix_per_cell, pix_per_cell),
                cells_per_block=(cell_per_block, cell_per_block),
                visualize=False, feature_vector=feature_vec, block_norm='L2-Hys')
    return hog_features

def extract_features(image, cspace='RGB', orient=9, pix_per_cell=8, 
                     spatial_size=(32, 32), hist_bins=32,
                     cell_per_block=2, hog_channel=0, spatial_feat=True,
                     hist_feat=True, hog_feat=True):
    feature_image = convert_color_rgb(image, cspace)
    
    img_features = []
    if spatial_feat:
        spatial_features = bin_spatial(feature_image, size=spatial_size)
        img_features.append(spatial_features)

    if hist_feat:
        hist_features = color_hist(feature_image, nbins=hist_bins)
        img_features.append(hist_features)
    
    if hog_feat:
        if hog_channel == 'ALL':
            hog_features = []
            for ch in range(feature_image.shape[2]):
                hog_features_ch = extract_hog_features(image[:, :, ch], orient, pix_per_cell, cell_per_block, feature_vec=True)
                hog_features.append(hog_features_ch)
            hog_features = np.ravel(hog_features)

        else:
            hog_features = extract_hog_features(feature_image[:, :, hog_channel], orient,
                                                pix_per_cell, cell_per_block, feature_vec=True)
        img_features.append(hog_features)
        
    return np.concatenate(img_features)
In [17]:
cspace = 'YUV' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9
pix_per_cell = 8
cell_per_block = 2
hog_channel = 'ALL' # Can be 0, 1, 2, or "ALL"
spatial_size=(16, 16)
hist_bins = 8
spatial_feat = True
hist_feat = True
hog_feat = True

Read images with cars and without cars

In [18]:
#CAR_IMAGES_PATH = ROOT_PATH/'data/vehicles/KITTI_extracted'
# CAR_IMAGES2_PATH = ROOT_PATH/'data/vehicles/GTI_MiddleClose'
# CAR_IMAGES3_PATH = ROOT_PATH/'data/vehicles/GTI_Left'
# CAR_IMAGES4_PATH = ROOT_PATH/'data/vehicles/GTI_Right'
# CAR_IMAGES5_PATH = ROOT_PATH/'data/vehicles/GTI_Far'
CAR_IMAGES_PATH = ROOT_PATH/'data/vehicles/all'

NOTCAR_IMAGES_PATH = ROOT_PATH/'data/non-vehicles/GTI'
NOTCAR_IMAGES2_PATH = ROOT_PATH/'data/non-vehicles/Extras'

print('Total car images: ', len(list(CAR_IMAGES_PATH.iterdir())))
# print('Total car images: ', len(list(CAR_IMAGES2_PATH.iterdir())))
# print('Total car images: ', len(list(CAR_IMAGES3_PATH.iterdir())))
# print('Total car images: ', len(list(CAR_IMAGES4_PATH.iterdir())))
# print('Total car images: ', len(list(CAR_IMAGES5_PATH.iterdir())))

print('Not car iMages: ', len(list(NOTCAR_IMAGES_PATH.iterdir())))
print('Not car iMages: ', len(list(NOTCAR_IMAGES2_PATH.iterdir())))
Total car images:  6920
Not car iMages:  3902
Not car iMages:  5069
In [19]:
%%time
SAMPLE_RATIO = 1

car_features = []
# car_files = (list(CAR_IMAGES_PATH.glob('*png')) + 
#              list(CAR_IMAGES2_PATH.glob('*png')) + 
#              list(CAR_IMAGES3_PATH.glob('*png')) +
#              list(CAR_IMAGES4_PATH.glob('*png')) +
#              list(CAR_IMAGES5_PATH.glob('*png')))

print('car features...')
car_files = list(CAR_IMAGES_PATH.glob('*png'))
num_files = len(car_files)
idx = np.random.choice(range(num_files), int(num_files*SAMPLE_RATIO))
for file in np.array(car_files)[idx]:
    img = read_image(file.as_posix())
    features =  extract_features(img, cspace=cspace, 
                            spatial_size=spatial_size, hist_bins=hist_bins, 
                            orient=orient, pix_per_cell=pix_per_cell, 
                            cell_per_block=cell_per_block, 
                            hog_channel=hog_channel, spatial_feat=spatial_feat, 
                            hist_feat=hist_feat, hog_feat=hog_feat)
    car_features.append(features)
plt.imshow(img)
plt.show()

print('notcar features...')
notcar_features = []
noncar_files = list(NOTCAR_IMAGES_PATH.glob('*png')) + list(NOTCAR_IMAGES2_PATH.glob('*png'))
num_files = len(noncar_files)
idx = np.random.choice(range(num_files), int(num_files*SAMPLE_RATIO))
for file in np.array(noncar_files)[idx]:
    img = read_image(file.as_posix())
    features =  extract_features(img, cspace=cspace, 
                            spatial_size=spatial_size, hist_bins=hist_bins, 
                            orient=orient, pix_per_cell=pix_per_cell, 
                            cell_per_block=cell_per_block, 
                            hog_channel=hog_channel, spatial_feat=spatial_feat, 
                            hist_feat=hist_feat, hog_feat=hog_feat)
    notcar_features.append(features)
plt.imshow(img)
plt.show()
car features...
notcar features...
CPU times: user 1min 22s, sys: 769 ms, total: 1min 23s
Wall time: 1min 23s
In [20]:
X = np.vstack((car_features, notcar_features)).astype(np.float64)
print('Shape of X: ', X.shape)
Shape of X:  (14299, 6084)
In [21]:
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))
print('Shape of y: ', y.shape)
Shape of y:  (14299,)

Random split

In [22]:
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=rand_state)
In [23]:
# Fit a per-column scaler
X_scaler = StandardScaler().fit(X_train)
X_train = X_scaler.transform(X_train)
X_test = X_scaler.transform(X_test)

Training

In [24]:
print('Using:',orient,'orientations',pix_per_cell,
    'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))
# Use a linear SVC 
svc = LinearSVC()
# Check the training time for the SVC
t=time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')
# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))
# Check the prediction time for a single sample
t=time.time()
n_predict = 3000
#print('My SVC predicts: ', svc.predict(X_test[0:n_predict]))
#print('For these',n_predict, 'labels: ', y_test[0:n_predict])
t2 = time.time()
print(round(t2-t, 5), 'Seconds to predict', n_predict,'labels with SVC')
Using: 9 orientations 8 pixels per cell and 2 cells per block
Feature vector length: 6084
25.45 Seconds to train SVC...
Test Accuracy of SVC =  0.9951
5e-05 Seconds to predict 3000 labels with SVC

Sliding windows

In [25]:
def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):
    # Make a copy of the image
    imcopy = np.copy(img)
    # Iterate through the bounding boxes
    for bbox in bboxes:
        # Draw a rectangle given bbox coordinates
        cv2.rectangle(imcopy, bbox[0], bbox[1], color, thick)
    # Return the image copy with boxes drawn
    return imcopy
In [26]:
def slide_window(img, x_start_stop=[0, None], y_start_stop=[0, None],
                 xy_window=(64, 64), xy_overlap=(0.5, 0.5)):
    if x_start_stop[1] == None:
        x_start_stop[1] = img.shape[1]
    if y_start_stop[1] == None:
        y_start_stop[1] = img.shape[0]
    
    xspan = x_start_stop[1] - x_start_stop[0]
    yspan = y_start_stop[1] - y_start_stop[0]
    
    nx_pix_per_step = np.int(xy_window[0]*(1-xy_overlap[0]))
    ny_pix_per_step = np.int(xy_window[1]*(1-xy_overlap[1]))
    
    nx_buffer = np.int(xy_window[0]*(xy_overlap[0]))
    ny_buffer = np.int(xy_window[1]*(xy_overlap[1]))
    
    # number of windows in x and y
    nx_windows = np.int((xspan-nx_buffer)/nx_pix_per_step)
    ny_windows = np.int((yspan-ny_buffer)/ny_pix_per_step)
    
    window_list = []
    for xs in range(nx_windows):
        for ys in range(ny_windows):
            startx = xs*nx_pix_per_step + x_start_stop[0]
            endx = startx + xy_window[0]
            starty = ys*ny_pix_per_step + y_start_stop[0]
            endy = starty + xy_window[1]
            
            window_list.append(((startx, starty), (endx, endy)))
    
    return window_list
In [27]:
test_image_path = ROOT_PATH/'images/test_images/test5.jpg'
test_image = read_image(test_image_path.as_posix())
In [28]:
windows = slide_window(test_image, xy_window=(64, 64), xy_overlap=(0.5, 0.5))

img = draw_boxes(test_image, windows)
plt.figure(figsize=(12,9))
plt.imshow(img)
Out[28]:
<matplotlib.image.AxesImage at 0x7f1827ef8a20>

Search and classify

In [29]:
# Define a function you will pass an image 
# and the list of windows to be searched (output of slide_windows())
def search_windows(img, windows, clf, scaler, cspace='RGB', 
                    spatial_size=(32, 32), hist_bins=32, 
                    hist_range=(0, 256), orient=9, 
                    pix_per_cell=8, cell_per_block=2, 
                    hog_channel=0, spatial_feat=True, 
                    hist_feat=True, hog_feat=True):

    #1) Create an empty list to receive positive detection windows
    on_windows = []
    #2) Iterate over all windows in the list
    for window in windows:
        #3) Extract the test window from original image
        test_img = cv2.resize(img[window[0][1]:window[1][1], window[0][0]:window[1][0]], (64, 64))      

        #4) Extract features for that window using single_img_features()
        features = extract_features(test_img, cspace=cspace, 
                            spatial_size=spatial_size, hist_bins=hist_bins, 
                            orient=orient, pix_per_cell=pix_per_cell, 
                            cell_per_block=cell_per_block, 
                            hog_channel=hog_channel, spatial_feat=spatial_feat, 
                            hist_feat=hist_feat, hog_feat=hog_feat)
        #5) Scale extracted features to be fed to classifier
        test_features = scaler.transform(np.array(features).reshape(1, -1))
        #6) Predict using your classifier
        prediction = clf.predict(test_features)
        #7) If positive (prediction == 1) then save the window
        if prediction == 1:
            #print(features.shape)
            #plt.imshow(test_img)
            #plt.show()
            on_windows.append(window)
    #8) Return windows for positive detections
    return on_windows
In [30]:
def slide_and_search(img, y_start_stop, classifier, scaler, xy_window, xy_overlap, cspace, 
                     spatial_size, hist_bins, orient, pix_per_cell, 
                     cell_per_block, hog_channel, spatial_feat=True, hist_feat=True, hog_feat=True):
    # crop the image
    windows = slide_window(img, xy_window=xy_window, xy_overlap=xy_overlap, y_start_stop=y_start_stop)
    hot_windows = search_windows(img, windows, svc, X_scaler, cspace=cspace,  
                        spatial_size=spatial_size, hist_bins=hist_bins, 
                        orient=orient, pix_per_cell=pix_per_cell, 
                        cell_per_block=cell_per_block, 
                        hog_channel=hog_channel, spatial_feat=spatial_feat, 
                        hist_feat=hist_feat, hog_feat=hog_feat) 
    return windows, hot_windows
In [31]:
def draw_windows(img, windows, hot_windows, vis_search=False):
    if vis_search:
        img = draw_boxes(img, windows, color=(242, 255, 191))
    img = draw_boxes(img, hot_windows, color=(216, 48, 10))
    
    return img
In [32]:
%%time
xy_window = (48, 48)
xy_overlap = (0.5, 0.5)
y_start_stop = (395, 500)
windows, hot_windows = slide_and_search(test_image, y_start_stop, svc, X_scaler, xy_window, xy_overlap, cspace=cspace,  
                        spatial_size=spatial_size, hist_bins=hist_bins, 
                        orient=orient, pix_per_cell=pix_per_cell, 
                        cell_per_block=cell_per_block, 
                        hog_channel=hog_channel, spatial_feat=spatial_feat, 
                        hist_feat=hist_feat, hog_feat=hog_feat)
result_image = draw_windows(test_image, windows, hot_windows, vis_search=True)
plt.figure(figsize=(12, 9))
plt.imshow(result_image)
CPU times: user 866 ms, sys: 224 µs, total: 867 ms
Wall time: 866 ms
In [33]:
%%time
xy_window = (64, 64)
xy_overlap = (0.4, 0.4)
y_start_stop = (395, 550)
windows, hot_windows = slide_and_search(test_image, y_start_stop, svc, X_scaler, xy_window, xy_overlap, cspace=cspace,  
                        spatial_size=spatial_size, hist_bins=hist_bins, 
                        orient=orient, pix_per_cell=pix_per_cell, 
                        cell_per_block=cell_per_block, 
                        hog_channel=hog_channel, spatial_feat=spatial_feat, 
                        hist_feat=hist_feat, hog_feat=hog_feat)
result_image = draw_windows(test_image, windows, hot_windows, vis_search=True)
plt.figure(figsize=(12, 9))
plt.imshow(result_image)
CPU times: user 601 ms, sys: 0 ns, total: 601 ms
Wall time: 600 ms
In [34]:
%%time
xy_window = (80, 80)
xy_overlap = (0.4, 0.4)
y_start_stop = (395, 560)
windows, hot_windows = slide_and_search(test_image, y_start_stop, svc, X_scaler, xy_window, xy_overlap, cspace=cspace,  
                        spatial_size=spatial_size, hist_bins=hist_bins, 
                        orient=orient, pix_per_cell=pix_per_cell, 
                        cell_per_block=cell_per_block, 
                        hog_channel=hog_channel, spatial_feat=spatial_feat, 
                        hist_feat=hist_feat, hog_feat=hog_feat)
result_image = draw_windows(test_image, windows, hot_windows, vis_search=True)
plt.figure(figsize=(12, 9))
plt.imshow(result_image)
CPU times: user 390 ms, sys: 167 µs, total: 390 ms
Wall time: 389 ms
In [35]:
%%time
xy_window = (92, 92)
xy_overlap = (0.4, 0.4)
y_start_stop = (400, 600)
windows, hot_windows = slide_and_search(test_image, y_start_stop, svc, X_scaler, xy_window, xy_overlap, cspace=cspace,  
                        spatial_size=spatial_size, hist_bins=hist_bins, 
                        orient=orient, pix_per_cell=pix_per_cell, 
                        cell_per_block=cell_per_block, 
                        hog_channel=hog_channel, spatial_feat=spatial_feat, 
                        hist_feat=hist_feat, hog_feat=hog_feat)
result_image = draw_windows(test_image, windows, hot_windows, vis_search=True)
plt.figure(figsize=(12, 9))
plt.imshow(result_image)
CPU times: user 273 ms, sys: 3.73 ms, total: 276 ms
Wall time: 275 ms
In [36]:
%%time
xy_window = (116, 116)
xy_overlap = (0.5, 0.5)
y_start_stop = (400, 700)
windows, hot_windows = slide_and_search(test_image, y_start_stop, svc, X_scaler, xy_window, xy_overlap, cspace=cspace,  
                        spatial_size=spatial_size, hist_bins=hist_bins, 
                        orient=orient, pix_per_cell=pix_per_cell, 
                        cell_per_block=cell_per_block, 
                        hog_channel=hog_channel, spatial_feat=spatial_feat, 
                        hist_feat=hist_feat, hog_feat=hog_feat)
result_image = draw_windows(test_image, windows, hot_windows, vis_search=True)
plt.figure(figsize=(12, 9))
plt.imshow(result_image)
CPU times: user 491 ms, sys: 286 µs, total: 491 ms
Wall time: 490 ms

Multiple detections and false positive

In [37]:
def add_heat(heatmap, bbox_list):
    # Iterate through list of bboxes
    for box in bbox_list:
        # Add += 1 for all pixels inside each bbox
        # Assuming each "box" takes the form ((x1, y1), (x2, y2))
        heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1

    # Return updated heatmap
    return heatmap# Iterate through list of bboxes
    
def apply_threshold(heatmap, threshold):
    # Zero out pixels below the threshold
    heatmap[heatmap <= threshold] = 0
    # Return thresholded map
    return heatmap

def draw_labeled_bboxes(img, labels):
    # Iterate through all detected cars
    for car_number in range(1, labels[1]+1):
        # Find pixels with each car_number label value
        nonzero = (labels[0] == car_number).nonzero()
        # Identify x and y values of those pixels
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        # Define a bounding box based on min/max x and y
        bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
        # Draw the box on the image
        cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6)
    # Return the image
    return img

# def filter_heatmap_with_thresh(heatmap, labels, thresh):
#     num_labels = 0
#     for car_number in range(1, labels[1]+1):
#         nonzero = (labels[0] == car_number).nonzero()
#         box_size = (np.max(nonzerox)-np.min(nonzerox))*(np.max(nonzeroy)-np.min(nonzeroy))
#         if box_size < thresh:
#             labels[0][nonzeros[0], nonzero[1]] = 0      
In [38]:
%%time
test_image_path = ROOT_PATH/'images/test_images/test6.jpg'
test_image = read_image(test_image_path.as_posix())

window_list = [
    (48, 0.5, 395, 500),
    (64, 0.5, 405, 520),
    (80, 0.5, 395, 560),
    #(92, 0.5, 400, 600),
    (116, 0.5, 400, 700)
]

threshold_detect = 3

def detect_cars(img, window_list, threshold_detect, classifier, 
                scaler, cspace, spatial_size, hist_bins, orient, pix_per_cell, 
                cell_per_block, hog_channel, spatial_feat, hist_feat, hog_feat):
    heat = np.zeros_like(test_image[:,:,0]).astype(np.float)
    
    for window, overlap, ystart, ystop in window_list:
        xy_window = (window, window)
        xy_overlap = (overlap, overlap)
        y_start_stop = (ystart, ystop)
        _, hot_windows = slide_and_search(img, y_start_stop, classifier, scaler, xy_window, xy_overlap, 
                                          cspace=cspace, spatial_size=spatial_size, hist_bins=hist_bins, 
                                          orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, 
                                          hog_channel=hog_channel, spatial_feat=spatial_feat, 
                                          hist_feat=hist_feat, hog_feat=hog_feat)
        heat = add_heat(heat, hot_windows)
    
    # apply the threhold
    heat = apply_threshold(heat, threshold_detect)
    
    # create heatmap and labels
    heatmap = np.clip(heat, 0, 255)
    labels = label(heatmap)
    
    return heatmap, labels
 

heatmap, labels = detect_cars(test_image, window_list, threshold_detect, svc, X_scaler, cspace=cspace,  
                              spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, 
                              pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, 
                              hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat,
                              hog_feat=hog_feat)
CPU times: user 2.36 s, sys: 4.01 ms, total: 2.36 s
Wall time: 2.36 s
In [39]:
for img_path in (ROOT_PATH/'images/test_images').glob('*jpg'):
    print(img_path)
    img = read_image(img_path.as_posix())
    heatmap, labels = detect_cars(img, window_list, threshold_detect, svc, X_scaler, cspace=cspace,  
                              spatial_size=spatial_size, hist_bins=hist_bins, orient=orient, 
                              pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, 
                              hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat,
                              hog_feat=hog_feat)
    print('number of labels: ', labels[1])
    draw_img = draw_labeled_bboxes(np.copy(img), labels)

    fig = plt.figure(figsize=(16,12))
    plt.subplot(121)
    plt.imshow(draw_img)
    plt.title('Car Positions')
    plt.subplot(122)
    plt.imshow(heatmap, cmap='hot')
    plt.title('Heat Map')
    fig.tight_layout()
    
    plt.show()
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test4.jpg
number of labels:  3
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test3.jpg
number of labels:  0
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test1.jpg
number of labels:  1
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test6.jpg
number of labels:  1
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test2.jpg
number of labels:  0
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test5.jpg
number of labels:  1

HOG Subsampling

In [40]:
# Define a single function that can extract features using hog sub-sampling and make predictions
def find_cars(img, cspace, cells_per_step, ystart, ystop, scale, svc, X_scaler, 
              orient, pix_per_cell, cell_per_block, spatial_size, hist_bins):
    
    draw_img = np.copy(img)
    #img = img.astype(np.float32)/255
    
    img_tosearch = img[ystart:ystop,:,:]
    ctrans_tosearch = convert_color_rgb(img_tosearch, cspace=cspace)
    if scale != 1:
        imshape = ctrans_tosearch.shape
        ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
        print('ctrans_tosearch shape: ', ctrans_tosearch.shape)
        
    ch1 = ctrans_tosearch[:,:,0]
    ch2 = ctrans_tosearch[:,:,1]
    ch3 = ctrans_tosearch[:,:,2]
    #ch1 = ctrans_tosearch[:,:,0]

    # Define blocks and steps as above
    nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1
    nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1 
    #nfeat_per_block = orient*cell_per_block**2
    
    # 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
    window = 64
    nblocks_per_window = (window // pix_per_cell) - cell_per_block + 1
    cells_per_step = cells_per_step  # Instead of overlap, define how many cells to step
    nxsteps = (nxblocks - nblocks_per_window) // cells_per_step + 1
    nysteps = (nyblocks - nblocks_per_window) // cells_per_step + 1
    
    # Compute individual channel HOG features for the entire image
    hog1 = extract_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog2 = extract_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog3 = extract_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
    #hog = extract_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
    
    for xb in range(nxsteps):
        for yb in range(nysteps):
            ypos = yb*cells_per_step
            xpos = xb*cells_per_step
            # Extract HOG for this patch
            hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))
            #hog_features = hog[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 

            xleft = xpos*pix_per_cell
            ytop = ypos*pix_per_cell

            # Extract the image patch
            subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (64, 64))
          
            # Get color features
            spatial_features = bin_spatial(subimg, size=spatial_size)
            hist_features = color_hist(subimg, nbins=hist_bins)

            # Scale features and make a prediction
            test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))      
            test_prediction = svc.predict(test_features)
            
            if test_prediction == 1:
                xbox_left = np.int(xleft*scale)
                ytop_draw = np.int(ytop*scale)
                win_draw = np.int(window*scale)
                cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),
                             (xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,0,255),6) 
                
    return draw_img
In [197]:
%%time
test_image_path = ROOT_PATH/'images/test_images/test3.jpg'
test_image = read_image(test_image_path.as_posix())

ystart = 402
ystop = 500
scale = 0.98
cells_per_step = 4
out_img = find_cars(test_image, cspace, cells_per_step, ystart, ystop, scale, svc,
                    X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)

plt.figure(figsize=(12, 9))
plt.imshow(out_img)
ctrans_tosearch shape:  (100, 1306, 3)
CPU times: user 297 ms, sys: 15.3 ms, total: 312 ms
Wall time: 285 ms
In [198]:
%%time

ystart = 390
ystop = 550
scale = 1
cells_per_step = 4
out_img = find_cars(test_image, cspace, cells_per_step, ystart, ystop, scale, svc,
                    X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)

plt.figure(figsize=(12, 9))
plt.imshow(out_img)
CPU times: user 392 ms, sys: 13.1 ms, total: 405 ms
Wall time: 389 ms
In [202]:
%%time
ystart = 400
ystop = 550
scale = 1.25
cells_per_step = 4
out_img = find_cars(test_image, cspace, cells_per_step, ystart, ystop, scale, svc,
                    X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)

plt.figure(figsize=(12, 9))
plt.imshow(out_img)
ctrans_tosearch shape:  (120, 1024, 3)
CPU times: user 239 ms, sys: 26.7 ms, total: 266 ms
Wall time: 243 ms
In [187]:
%%time
ystart = 390
ystop = 600
scale = 1.5
cells_per_step = 6
out_img = find_cars(test_image, cspace, cells_per_step, ystart, ystop, scale, svc,
                    X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)

plt.figure(figsize=(12, 9))
plt.imshow(out_img)
ctrans_tosearch shape:  (140, 853, 3)
CPU times: user 210 ms, sys: 25.5 ms, total: 236 ms
Wall time: 212 ms
In [188]:
%%time
ystart = 395
ystop = 600
scale = 1.75
cells_per_step = 6
out_img = find_cars(test_image, cspace, cells_per_step, ystart, ystop, scale, svc,
                    X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)

plt.figure(figsize=(12, 9))
plt.imshow(out_img)
ctrans_tosearch shape:  (117, 731, 3)
CPU times: user 181 ms, sys: 3.42 ms, total: 185 ms
Wall time: 168 ms
In [189]:
%%time
ystart = 390
ystop = 650
scale = 2.25
cells_per_step = 4
out_img = find_cars(test_image, cspace, cells_per_step, ystart, ystop, scale, svc,
                    X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)

plt.figure(figsize=(12, 9))
plt.imshow(out_img)
ctrans_tosearch shape:  (115, 568, 3)
CPU times: user 137 ms, sys: 25.1 ms, total: 162 ms
Wall time: 143 ms
In [208]:
%%time
ystart = 395
ystop = 660
scale = 2.5
cells_per_step = 2
out_img = find_cars(test_image, cspace, cells_per_step, ystart, ystop, scale, svc,
                    X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)

plt.figure(figsize=(12, 9))
plt.imshow(out_img)
ctrans_tosearch shape:  (106, 512, 3)
CPU times: user 231 ms, sys: 9.93 ms, total: 241 ms
Wall time: 223 ms

Subsampling with multiple windows

In [157]:
def search_windows_with_scale(img, scale, cells_per_step, ystart, ystop, classifier, 
                              scaler, cspace, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins):
    hot_windows = []
    draw_img = np.copy(img)
    
    # crate an empty list to receive positive detection windows
    img_tosearch = img[ystart:ystop,:,:]
    ctrans_tosearch = convert_color_rgb(img_tosearch, cspace=cspace)
    if scale != 1:
        imshape = ctrans_tosearch.shape
        ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
    #print('ctrans_tosearch shape: ', ctrans_tosearch.shape)
        
    ch1 = ctrans_tosearch[:,:,0]
    ch2 = ctrans_tosearch[:,:,1]
    ch3 = ctrans_tosearch[:,:,2]

    # Define blocks and steps as above
    nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1
    nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1 
    #nfeat_per_block = orient*cell_per_block**2
    
    # 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
    window_size = 64
    nblocks_per_window = (window_size // pix_per_cell) - cell_per_block + 1
    cells_per_step = cells_per_step  # Instead of overlap, define how many cells to step
    nxsteps = (nxblocks - nblocks_per_window) // cells_per_step + 1
    nysteps = (nyblocks - nblocks_per_window) // cells_per_step + 1
    
    # Compute individual channel HOG features for the entire image
    #hog = extract_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog1 = extract_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog2 = extract_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog3 = extract_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
    
    for xb in range(nxsteps):
        for yb in range(nysteps):
            ypos = yb*cells_per_step
            xpos = xb*cells_per_step
            # Extract HOG for this patch
            #hog_features = hog[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))

            xleft = xpos*pix_per_cell
            ytop = ypos*pix_per_cell

            # Extract the image patch
            subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window_size, xleft:xleft+window_size], (64, 64))
          
            # Get color features
            spatial_features = bin_spatial(subimg, size=spatial_size)
            hist_features = color_hist(subimg, nbins=hist_bins)

            # Scale features and make a prediction
            #test_features = scaler.transform(np.hstack((spatial_features, hog_features)).reshape(1, -1))    
            test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))
            test_prediction = classifier.predict(test_features)
            
            if test_prediction == 1:
                xbox_left = np.int(xleft*scale)
                ytop_draw = np.int(ytop*scale)
                win_draw = np.int(window_size*scale)
                hot_window = ((xbox_left, ytop_draw+ystart), (xbox_left+win_draw, ytop_draw+win_draw+ystart))
                hot_windows.append(hot_window)
                
    return hot_windows
In [209]:
scales = [
    #(0.9,  2, 410, 500),
    (1,    4, 390, 550), 
    (1.25, 4, 400, 550),
    (1.75, 6, 395, 650),
    (2.25, 4, 390, 650),
    (2.5,  2, 395, 660)
]

threshold = 3
In [210]:
def detect_with_scales(img, scales, threshold, classifier, scaler, cspace, orient, 
                       pix_per_cell, cell_per_block, spatial_size, hist_bins):
    # create an empty array to store heatmaps
    heat = np.zeros_like(img[:,:,0]).astype(np.float)
    
    for scale, cells_per_step, ystart, ystop in scales:
        hot_windows = search_windows_with_scale(img, scale, cells_per_step, ystart, ystop, svc,
                    scaler, cspace, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)
        heat = add_heat(heat, hot_windows)
    
    # apply the threshold 
    heatmap = apply_threshold(heat, threshold)
    labels = label(heatmap)
    
    return heatmap, labels
In [211]:
%%time
test_image_path = ROOT_PATH/'images/test_images/test3.jpg'
test_image = read_image(test_image_path.as_posix())

heatmap, labels = detect_with_scales(test_image, scales, threshold, svc, X_scaler, cspace, 
                                     orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)
CPU times: user 990 ms, sys: 49.5 ms, total: 1.04 s
Wall time: 954 ms
In [212]:
for img_path in (ROOT_PATH/'images/test_images').glob('*jpg'):
    print(img_path)
    img = read_image(img_path.as_posix())
    heatmap, labels = detect_with_scales(img, scales, threshold, svc, X_scaler, cspace, 
                                     orient, pix_per_cell, cell_per_block, spatial_size, hist_bins)
    print('number of labels: ', labels[1])
    draw_img = draw_labeled_bboxes(np.copy(img), labels)

    fig = plt.figure(figsize=(16,12))
    plt.subplot(121)
    plt.imshow(draw_img)
    plt.title('Car Positions')
    plt.subplot(122)
    plt.imshow(heatmap, cmap='hot')
    plt.title('Heat Map')
    fig.tight_layout()
    
    plt.show()
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test4.jpg
number of labels:  1
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test3.jpg
number of labels:  0
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test1.jpg
number of labels:  0
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test6.jpg
number of labels:  1
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test2.jpg
number of labels:  0
/home/downloads/carnd-p5-vehicle-detection/images/test_images/test5.jpg
number of labels:  2